import json
import time

import scrapy
from spidertools.utils.time_utils import get_current_date

from commonresources.spider_items.jiangsu.xuzhou.items import XuZhouGongGongZiYuanJiaoYiPingTaiItem
from spidertools.common_pipeline.base_item import convert_dict


class XuZhouGongGongZiYuanJiaoYiPingTaiSpider(scrapy.Spider):
    """
    徐州公共资源交易平台 交易信息列表页爬虫
     http://www.xzggzy.com.cn
    """
    name = 'XuZhouGongGongZiYuanJiaoYiPingTai'
    name_zh = '徐州公共资源交易平台'
    province = "江苏"
    city = '徐州'
    allowed_domains = ['www.xzggzy.com.cn']
    start_urls = [
        'http://www.xzggzy.com.cn/services/XzsJsggWebservice/getListByCount?response=application/json&&categorynum=&bianhao=&xmmc=&diqutype=']

    def __init__(self, full_dose=False):
        """
            :param full_dose: 是否全量爬取，默认为false
        """
        self.browser_cookie = {}
        self.page_count = -1
        self.convert_dict = convert_dict
        self.full_dose = full_dose
        super().__init__()

    def parse(self, response):
        page_obj = json.loads(response.text)
        self.page_count = int(page_obj['return']) // 15 + 1
        next_page_url = f'http://www.xzggzy.com.cn/services/XzsJsggWebservice/getList?response=application/json&pageIndex=0&pageSize=15&&categorynum=&bianhao=&xmmc=&diqutype='
        yield scrapy.Request(url=next_page_url, cookies=self.browser_cookie, meta={'page': 0, 'need_break': False},
                             callback=self.handle_response,dont_filter=True)

    def handle_response(self, response):
        obj_json = json.loads(response.text)
        obj_json_page = json.loads(obj_json['return'])
        rows_info_list = obj_json_page['Table']
        for row in rows_info_list:
            item = dict()
            item['construction_type'] = row['jyfl']
            item['project_area'] = row['city']
            item['release_time'] = row['postdate']
            if not self.full_dose and item['release_time'] != get_current_date() :# and item['release_time'] != '2020-10-28':
                response.request.meta['need_break'] = True
            item['origin_url'] = "http://www.xzggzy.com.cn" + row['href']
            item['announcement_title'] = row['title']
            yield scrapy.Request(url=item['origin_url'], meta=item, cookies=self.browser_cookie, callback=self.handle_next_html)
        if not response.request.meta['need_break']:
            page = response.request.meta['page'] + 1
            if page < self.page_count:
                next_page_url = f'http://www.xzggzy.com.cn/services/XzsJsggWebservice/getList?response=application/json&pageIndex={page}&pageSize=15&&categorynum=&bianhao=&xmmc=&diqutype='
                yield scrapy.Request(url=next_page_url, cookies=self.browser_cookie,
                                     meta={'page': page, 'need_break': False},
                                     callback=self.handle_response, dont_filter=True)

    def handle_next_html(self, response):
        item = XuZhouGongGongZiYuanJiaoYiPingTaiItem()
        item['project_area'] = response.request.meta['project_area']
        item['construction_type'] = response.request.meta['construction_type']
        item['release_time'] = response.request.meta['release_time']
        item['origin_url'] = response.request.meta['origin_url']
        item['announcement_title'] = response.request.meta['announcement_title']
        item['html'] = response.text
        item['source_type'] = self.name_zh
        item['is_parsed'] = 0
        item['province'] = self.province
        item['city'] = self.city
        yield item
